package cn.gzsendi.itmp.itmpanalysisserver

import org.apache.spark.{SparkConf, SparkContext}

/**
  * 单词行数计算
  */
object WordCount {

  def main(args: Array[String]): Unit = {
    val logFile = "data/wordcount.txt" // Should be some file on your system
    val conf = new SparkConf().setMaster("local[6]").setAppName("Simple Application")
    val sc = new SparkContext(conf)
    val rdd1 = sc.textFile(logFile, 2).cache()
    // 拆分文档
    val words = rdd1.flatMap(line => line.split(" "))
    // 每个单词指定一个词频1
    val counts = words.map(word => (word, 1))
    // 聚合
    val result = counts.reduceByKey { (x, y) => x + y }
    // 遍历集合
    result.foreach(item => println(item))
  }

}
